# -*-coding:utf8-*-
#-*-coding:utf8-*-

import re

import jieba.posseg
from bs4 import BeautifulSoup
def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        pass

    try:
        import unicodedata
        unicodedata.numeric(s)
        return True
    except (TypeError, ValueError):
        pass

    return False

def match_name(column_str):
    if isinstance(column_str, unicode):
        pass
    else:
        column_str = unicode(column_str, 'utf-8')
    p = re.compile( u'^(\u738b|\u674e|\u5f20|\u5218|\u9648|\u6768|\u9ec4|\u8d75|\u5434|\u5468|\u5f90|\u5b59|\u6731|\u6797|\u94b1|\u90d1|\u5b8b|\u4f55|\u9a6c|\u7f57|\u90ed|\u9ad8|\u6881|\u8c22|\u51af|\u97e9|\u5510|\u848b|\u8463|\u5f6d|\u8881|\u91d1|\u53f6|\u9b4f|\u8427|\u5415|\u8521|\u66f9|\u675c|\u859b|\u6c88|\u5362|\u8d3e|\u5b5f|\u59da|\u8bb8|\u7a0b|\u9093|\u53f2|\u6234|\u6bb5|\u4faf|\u79e6|\u90b1|\u59dc|\u7530|\u8c2d|\u77f3|\u6bdb|\u960e|\u4f59|\u4e8e|\u987e|\u8303|\u949f|\u4e01|\u590f|\u82cf|\u66fe|\u6bb5|\u5e9e|\u5c39|\u4e07|\u4e54|\u8d3a|\u5d14|\u6c6a|\u4efb|\u9676|\u6d2a|\u5ed6|\u5b54|\u989c|\u767d|\u7eaa|\u8042|\u6b66|\u83ab|\u90b9|\u90dd|\u6c5f|\u4e25|\u9c81|u5e38)[\u4e00-\u9fa5]{1,2}$')
    flag = False
    isstate = p.match(column_str)
    if isstate:
        flag=True
    return flag

def match_address(column_str):
    if isinstance(column_str, unicode):
        pass
    else:
        column_str = unicode(column_str, 'utf-8')
    p = re.compile(u'^[\u4e00-\u9fa5]*.*(\u7701|\u5e02|\u53bf|\u533a|\u6751|\u9547|\u8857|\u8def|\u5e7f\u4e1c|\u5c71\u4e1c|\u6cb3\u5357|\u56db\u5ddd|\u6c5f\u82cf|\u6cb3\u5317|\u6e56\u5357|\u5b89\u5fbd|\u6e56\u5317|\u6d59\u6c5f|\u5e7f\u897f|\u4e91\u5357|\u6c5f\u897f|\u8fbd\u5b81|\u798f\u5efa|\u9ed1\u9f99\u6c5f|\u5c71\u897f|\u9655\u897f|\u5317\u4eac|\u4e0a\u6d77|\u5929\u6d25|\u91cd\u5e86|\u8d35\u5dde|\u5409\u6797|\u8fbd\u5b81|\u5185\u8499\u53e4|\u7518\u8083|\u65b0\u7586|\u897f\u85cf|\u53f0\u6e7e|\u6d77\u5357)')
    flag=False
    isstate = p.search(column_str)
    if isstate:
        flag = True
    return flag

def match_marry(column_str):
    if isinstance(column_str, unicode):
        pass
    else:
        column_str = unicode(column_str, 'utf-8')
    marry=u"已婚 未婚 婚姻 离异 再婚 marry unmarried married divorced Unmarried Married Divorced"
    prelist=marry.split(" ")
    flag=False
    for each in prelist:
        if each in column_str:
          flag=True
          break
    return flag

def match_hotel(column_str):
    if isinstance(column_str, unicode):
        pass
    else:
        column_str = unicode(column_str, 'utf-8')
    hotel=u"酒店 旅店 旅馆 招待所 民宿 hotel Hotel"
    prelist = hotel.split(" ")
    flag = False
    for each in prelist:
        if each in column_str:
            flag = True
            break
    return flag
def match_education(column_str):
    if isinstance(column_str, unicode):
        pass
    else:
        column_str = unicode(column_str, 'utf-8')
    names=u"本科 大专 高中 职高 中专 初中 研究生 硕士 博士 学士 大学"
    prelist = names.split(" ")
    flag = False
    if len(column_str)<20:
        for each in prelist:
            if each in column_str:
                flag = True
                break
    return flag
#
# def match_college(column_str):
#     if isinstance(column_str, unicode):
#         pass
#     else:
#         column_str = unicode(column_str, 'utf-8')
#     schools=u"大学 学院 学校 技校 高级中学 高中 初级中学 初中 培训中心 college univercity College Univercity institute school"
#     prelist =schools.split(" ")
#     flag = False
#     if len(column_str)>11:
#         for each in prelist:
#             if each in column_str:
#                 flag = True
#                 break
#     return flag


def match_ip(column_str):
    p = re.compile(r'^((?:(2[0-4]\d)|(25[0-5])|([01]?\d\d?))\.){3}(?:(2[0-4]\d)|(255[0-5])|([01]?\d\d?))')
    flag = False
    isstate = p.match(column_str)
    if isstate:
        flag = True
    return flag

def match_mob(column_str):
  p=re.compile("^1(3|4|5|7|8)\d{9}$")
  flag = False
  isstate = p.match(column_str)
  if isstate:
      flag = True
  return flag

def match_phone(column_str):
    if isinstance(column_str, unicode):
        pass
    else:
        column_str = unicode(column_str, 'utf-8')
    p = re.compile( u'^([0+]?86-|\([0+]?86\))?((0?\d{2}-|\(0?\d{2}-\))\d{7,8}|(0?\d{3}-|\(0?\d{3}\))\d{7,8})([-转]\d{1,8})?')
    flag = False
    isstate = p.match(column_str)
    if isstate:
      flag = True
    return flag

def match_email(column_str):
    # p = re.compile(r'^[-\w]+((-\w+)|(\.\w+))*@[A-Za-z0-9]+((\.|-)[A-Za-z0-9]+)*\.[A-Za-z0-9]+$')
    p = re.compile(r'[-\w]+((-\w+)|(\.\w+))*@[A-Za-z0-9]+((\.|-)[A-Za-z0-9]+)*\.[A-Za-z0-9]+')
    flag = False
    isstate = p.search(column_str)
    if isstate:
      flag = True
    return flag


def match_url(column_str):
    p = re.compile(
    r'((?:http|https|ftp|ssh|git|mailto|file|telnet|news)?://)?' # http:// or https://
    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
    r'localhost|' # localhost...
    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
    r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
    r'(?::\d+)?' # optional port
    r'(?:/?|[/?]\S+)', re.IGNORECASE)
    flag = False
    isstate = p.search(column_str)
    if isstate:
      flag = True
    return flag

def match_date(column_str):
    # p1 = re.compile(r'^((16|17|18|19|20|21)\d{2})[-/.](\d{1,2})[-/.](\d{1,2})$')
    # p2 = re.compile(r'^(\d{1,2})[-/.](\d{1,2})[-/.]((16|17|18|19|20|21)\d{2})$')
    p1 = re.compile(r'((16|17|18|19|20|21)\d{2})[-/.](\d{1,2})[-/.](\d{1,2})')
    p2 = re.compile(r'(\d{1,2})[-/.](\d{1,2})[-/.]((16|17|18|19|20|21)\d{2})')
    flag = False
    isstate1 = p1.search(column_str)
    isstate2 = p2.search(column_str)
    if isstate1 or isstate2:
      flag = True
    return flag


def match_id(column_str):
    # p1=re.compile(r'^[1-9]\d{7}((0\d)|(1[0-2]))(([0|1|2]\d)|3[0-1])\d{3}$')
    p2=re.compile(r'^[1-9]\d{5}([1][9][2-9]|[2][0][0])\d{1}((0\d)|(1[0-2]))(([0|1|2]\d)|3[0-1])\d{3}([0-9]|X)$')
    # p2=re.compile(r'^[1-9]\d{5}[1-9]\d{3}((0\d)|(1[0-2]))(([0|1|2]\d)|3[0-1])\d{3}([0-9]|X)$')
    flag=False
    # isstate1=p1.match(column_str)
    isstate2=p2.match(column_str)
    if isstate2:
      flag = True
    return flag

def match_card(column_str):
    try:
        card_str=column_str.replace(" ","")
        cardlen=len(card_str)

        if cardlen>15 and cardlen<20:
            digits = [int(x) for x in reversed(str(card_str))]
            check_sum = sum(digits[::2]) + sum((dig // 10 + dig % 10) for dig in [2 * el for el in digits[1::2]])
            return check_sum % 10 == 0
        else:
            return False
    except:
        return False

# def match_money(column_str):
#     p = re.compile(r'^(([1-9]\d{0,9})|0)(\.\d{1,2})?$')
#     flag = False
#     isstate = p.match(column_str)
#     if isstate:
#       flag = True
#     return flag



rule1 = {"level": u"绝密", "category": "name", "rule": match_name}
rule2 = {"level": u"绝密", "category": "address", "rule": match_address}
rule3 = {"level": u"绝密", "category": "hotel", "rule": match_hotel}
rule5 = {"level": u"秘密", "category": "Education", "rule": match_education}
rule7={"level": u"绝密", "category": "ID", "rule": match_id}
rule9={"level": u"绝密", "category": "IP", "rule": match_ip}
rule10={"level": u"秘密", "category": "data", "rule": match_date}
rule11={"level": u"机密", "category": u"mob", "rule": match_mob}
rule12={"level": u"机密", "category": u"phone", "rule": match_phone}
rule13={"level": u"秘密", "category": u"URL", "rule": match_url}
rule15={"level": u"绝密", "category": u"card", "rule": match_card}

rules1=[rule3,rule5,rule7,rule9,rule10,rule11,rule12,rule13,rule15]
rules2=[rule1,rule14]

def vaild_sentive_columns(tupleTemp,label):
    try:
        if isinstance(tupleTemp, unicode):
            v=tupleTemp
        else:
            v=str(tupleTemp)
        for rule in rules:
            if rule['rule'](v):
                label.append(rule['category'])
        return label

    except UnicodeEncodeError:
        print "UnicodeEncodeError"
        print v
    except TypeError:
        print 'TypeError'
        print v
        print type(v)



def chai_iter(senlist, ad, dd):
    # try:
    for key, value in dd.items():
        pre=ad
        ad=ad + "." + str(key)
        types=recogtype(value)
        content=""
        if isinstance(value, unicode):
            content=value.encode("utf-8")

        try:
            content=str(value)
        except:
            pass

        if isinstance(value, dict) == False:
            if key == "body":
                soup=BeautifulSoup(value)
                senlist=analysishtml(senlist, ad, soup)
                ad=pre

            if isinstance(value, list):
                try:
                    for each in value:
                        chai_iter(senlist, ad, each)
                        ad=pre

                except Exception, e:
                    label=vaild_sentive_columns(content)
                    if len(label) > 0:
                        senone={"path": ad, "label": label, "type": types}
                        senlist.append(senone)
                    ad=pre
            else:
                label=vaild_sentive_columns(content)
                if len(label) > 0:
                    senone={"path": ad, "label": label, "type": types}
                    senlist.append(senone)
                ad=pre

        else:
            chai_iter(senlist, ad, value)
            ad=pre
    return senlist


def analysishtml(senlist, ad, soup):
    try:
        for child in soup.children:
            pre=ad
            types=type(child)
            try:
                cc=child.content
            except:
                cc=""
            if cc != "":
                ad+=unicode(child).split(">")[0] + ">"
                analysishtml(senlist, ad, child)
                ad=pre
            else:
                content=unicode(child)
                firstbodywords=jieba.posseg.cut(content)
                label=[]

                for i in firstbodywords:
                    label=vaild_sentive_columns(i.word,label)
                    if i.flag == "nr":
                        if len(i.word) > 1 and len(i.word) < 4 :
                            print i.word
                   # print i.word

                # if len(label) > 0:
                #     print "path:" + ad
                #     print "label:"
                #     print label
                #     senone={"path": ad, "label": label, "type": types}
                #     senlist.append(senone)
    except:
        pass
    return senlist


def recogtype(dt):
    if isinstance(dt, str):
        type="String"
    elif isinstance(dt, unicode):
        type="String"
    elif isinstance(dt, dict):
        type="json"
    elif isinstance(dt, long):
        type="long"
    elif isinstance(dt, float):
        type="float"
    elif isinstance(dt, int):
        type="int"
    else:
        type=type(dt)
    return type


def config(dl, data):
    api={"id": "Userinfo", "name": "用户信息接口"}
    reqinfo={}
    rspinfo={}

    rsp=dl.get("rsp")
    rspheader=rsp.get("header")
    req=dl.get("req")
    preurl=req.get("url")
    method=req.get("method")
    reqheader=req.get("header")

    url=preurl
    urlparams=[]
    if "?" in preurl:
        url=preurl.split("?")[0]
        pathparam=preurl.split("?")[1].split("&")
        for each in pathparam:
            urlkey=each.split(":")[0]
            urltype=type(each.split(":")[1])
            urlparam={"name": urlkey, "type": urltype}
            urlparams.append(urlparam)

    reqinfo["url"]=url
    headerParamer=[]
    for (d, x) in reqheader.items():
        name=d
        type=recogtype(x)
        header={"name": name, "type": type}
        headerParamer.append(header)

    when={}
    for (d, x) in rspheader.items():
        name=str(d)
        when[name]=x

    reqinfo["headerParamer"]=headerParamer
    reqinfo["pathParams"]=urlparams
    reqinfo["method"]=method
    rspinfo["data"]=data
    rspinfo["when"]=when
    api["request"]=reqinfo
    api["response"]=rspinfo
    print api


if __name__ == '__main__':
    # client=MongoClient('106.14.159.159', 27017)
    # db=client.test
    # collection=db.supeittt
    f=open("11.html", "r")
    ff=f.read()
    senlist=[]
    ad="/"
    soup=BeautifulSoup(ff)
    senlist=analysishtml(senlist, ad, soup)
    print senlist
    # for dd in collection.find():
    #     print dd
    #     senlist=[]
    #     ad="/"
    #     senlist=analysishtml(senlist, ad, dd)
    #     print senlist





